In [1]:
import tensorflow as tf
import numpy as np
from tensorflow import data
import shutil
from datetime import datetime
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt

from tensorflow.contrib.learn import learn_runner
from tensorflow.contrib.learn import make_export_strategy

print(tf.__version__)


/Users/khalidsalama/anaconda/lib/python3.6/importlib/_bootstrap.py:205: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6
  return f(*args, **kwds)
1.4.0

Steps to use the TF Experiment API

  1. Define dataset metadata
  2. Define data input function to read the data from the source (csv) + apply pre-processing
  3. Create an estimator (KMeans) with parameters
  4. Run Experiment with estimator, training data, and serving function
  5. Predict cluster index of each instance

In [2]:
train_data_files = ['data/train-data.csv']
test_data_files = ['data/test-data.csv']

model_name = 'clust-model-02'

resume = False
train = True
preprocess_features = False
extend_feature_colums = False

1. Define Dataset Metadata


In [3]:
HEADER = ['key', 'x1', 'x2', 'x3', 'cluster']  
HEADER_DEFAULTS = [[0], [0.0], [0.0], [0.0], ['NA']]

FEATURE_NAMES = ['x1', 'x2', 'x3']  

UNUSED_FEATURE_NAMES = list(set(HEADER) - set(FEATURE_NAMES))

print("Header: {}".format(HEADER))
print("Input Features: {}".format(FEATURE_NAMES))
print("Unused Features: {}".format(UNUSED_FEATURE_NAMES))


Header: ['key', 'x1', 'x2', 'x3', 'cluster']
Input Features: ['x1', 'x2', 'x3']
Unused Features: ['cluster', 'key']

2. Define Data Input Function

a. parsing and preprocessing logic


In [4]:
def parse_csv_row(csv_row):
    
    columns = tf.decode_csv(csv_row, record_defaults=HEADER_DEFAULTS)
    columns = [tf.expand_dims(tensor, -1) for tensor in columns]
    features = dict(zip(HEADER, columns))
    
    for column in UNUSED_FEATURE_NAMES:
        features.pop(column)
    
    return features

def process_features(features):
    
    if preprocess_features:
        features = features
    
    return features

b. data pipeline input function


In [5]:
def csv_input_fn(file_names, mode=tf.estimator.ModeKeys.TRAIN, 
                 skip_header_lines=0, 
                 num_epochs=None, 
                 batch_size=200):
    
    shuffle = False
    
    print("")
    print("* data input_fn:")
    print("================")
    print("Input file(s): {}".format(file_names))
    print("Batch size: {}".format(batch_size))
    print("Epoch Count: {}".format(num_epochs))
    print("Mode: {}".format(mode))
    print("Shuffle: {}".format(shuffle))
    print("================")
    print("")

    dataset = data.TextLineDataset(filenames=file_names)
    dataset = dataset.skip(skip_header_lines)
    
    dataset = dataset.batch(batch_size)
    dataset = dataset.map(lambda csv_row: parse_csv_row(csv_row))
    dataset = dataset.map(lambda features: process_features(features))
    
    dataset = dataset.repeat(num_epochs)
    iterator = dataset.make_one_shot_iterator() 
    
    features = iterator.get_next()
    return features, None

In [6]:
features, _ = csv_input_fn(file_names=train_data_files)
print("Feature read from CSV: {}".format(list(features.keys())))


* data input_fn:
================
Input file(s): ['data/train-data.csv']
Batch size: 200
Epoch Count: None
Mode: train
Shuffle: False
================

Feature read from CSV: ['x1', 'x2', 'x3']

3. Create Estimator


In [7]:
def create_estimator(run_config, hparams):
    
    estimator = tf.contrib.learn.KMeansClustering(
        num_clusters = hparams.num_clusters,
        initial_clusters= tf.contrib.factorization.RANDOM_INIT,
        distance_metric= tf.contrib.factorization.SQUARED_EUCLIDEAN_DISTANCE,
        use_mini_batch=True,
        mini_batch_steps_per_iteration=1,
        kmeans_plus_plus_num_retries=10,
        relative_tolerance=None,
        config= run_config
    )

    print("")
    print("Estimator Type: {}".format(type(estimator)))
    print("")
    
    return estimator

4. Run Experiment

a. Create a serving function


In [8]:
def csv_serving_input_fn():
    
    SERVING_HEADER = ['x1','x2','x3']
    SERVING_HEADER_DEFAULTS = [[0.0], [0.0], [0.0]]
    
    rows_string_tensor = tf.placeholder(dtype=tf.string,
                                         shape=[None],
                                         name='csv_rows')
    
    receiver_tensor = {'csv_rows': rows_string_tensor}

    row_columns = tf.expand_dims(rows_string_tensor, -1)
    columns = tf.decode_csv(row_columns, record_defaults=SERVING_HEADER_DEFAULTS)
    columns = [tf.expand_dims(tensor, -1) for tensor in columns]
    features = dict(zip(SERVING_HEADER, columns))

    return tf.contrib.learn.InputFnOps(
        process_features(features),
        None,
        receiver_tensor
    )

b. Create Experiment Function


In [9]:
def generate_experiment_fn(**experiment_args):

    def _experiment_fn(run_config, hparams):

        train_input_fn = lambda: csv_input_fn(
            train_data_files,
            mode = tf.contrib.learn.ModeKeys.TRAIN,
            num_epochs=hparams.num_epochs,
            batch_size=hparams.batch_size*10
        )
        
        eval_input_fn = lambda: csv_input_fn(
            train_data_files,
            mode = tf.contrib.learn.ModeKeys.EVAL,
            num_epochs=1,
            batch_size=hparams.batch_size
        )

        estimator = create_estimator(run_config, hparams)

        return tf.contrib.learn.Experiment(
            estimator,
            train_input_fn=train_input_fn,
            eval_input_fn=eval_input_fn,
            eval_steps=None,
            **experiment_args
        )

    return _experiment_fn

c. Set HParam and RunConfig


In [10]:
hparams  = tf.contrib.training.HParams(
    num_epochs = 1000,
    batch_size = 500,
    num_clusters=3
)

model_dir = 'trained_models/{}'.format(model_name)

run_config = tf.contrib.learn.RunConfig(
    save_checkpoints_steps=100,
    tf_random_seed=19850610,
    model_dir=model_dir
)

print(run_config.model_dir)


trained_models/clust-model-02

d. Run Experiment


In [11]:
if not resume:
    print("Removing previous artifacts...")
    shutil.rmtree(model_dir, ignore_errors=True)
else:
    print("Resuming training...") 

if train:
    tf.logging.set_verbosity(tf.logging.INFO)
    
    time_start = datetime.utcnow() 
    print("Experiment started at {}".format(time_start.strftime("%H:%M:%S")))
    print(".......................................") 
    
    learn_runner.run(
        experiment_fn=generate_experiment_fn(

            export_strategies=[make_export_strategy(
                csv_serving_input_fn,
                exports_to_keep=1
            )]
        ), # not executing export_savedmodel()
        run_config=run_config,
        schedule="train", #evaluate function is borken, thus not using train_and_evaluate
        hparams=hparams
    )
    
    time_end = datetime.utcnow() 
    print(".......................................")
    print("Experiment finished at {}".format(time_end.strftime("%H:%M:%S")))
    print("")
    time_elapsed = time_end - time_start
    print("Experiment elapsed time: {} seconds".format(time_elapsed.total_seconds()))


Removing previous artifacts...
Experiment started at 18:48:39
.......................................
WARNING:tensorflow:RunConfig.uid (from tensorflow.contrib.learn.python.learn.estimators.run_config) is experimental and may change or be removed at any time, and without warning.
INFO:tensorflow:Using config: {'_task_type': None, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x120f46978>, '_master': '', '_num_ps_replicas': 0, '_num_worker_replicas': 0, '_environment': 'local', '_is_chief': True, '_evaluation_master': '', '_tf_config': gpu_options {
  per_process_gpu_memory_fraction: 1
}
, '_tf_random_seed': 19850610, '_save_summary_steps': 100, '_save_checkpoints_secs': None, '_log_step_count_steps': 100, '_session_config': None, '_save_checkpoints_steps': 100, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': 'trained_models/clust-model-02'}

Estimator Type: <class 'tensorflow.contrib.learn.python.learn.estimators.kmeans.KMeansClustering'>

WARNING:tensorflow:RunConfig.uid (from tensorflow.contrib.learn.python.learn.estimators.run_config) is experimental and may change or be removed at any time, and without warning.

* data input_fn:
================
Input file(s): ['data/train-data.csv']
Batch size: 5000
Epoch Count: 1000
Mode: train
Shuffle: False
================

WARNING:tensorflow:From /Users/khalidsalama/anaconda/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/estimators/kmeans.py:120: get_global_step (from tensorflow.contrib.framework.python.ops.variables) is deprecated and will be removed in a future version.
Instructions for updating:
Please switch to tf.train.get_global_step
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Saving checkpoints for 1 into trained_models/clust-model-02/model.ckpt.
INFO:tensorflow:loss = 127244.0, step = 1
INFO:tensorflow:Saving checkpoints for 101 into trained_models/clust-model-02/model.ckpt.
INFO:tensorflow:global_step/sec: 66.1581
INFO:tensorflow:loss = 54847.3, step = 101 (1.513 sec)
INFO:tensorflow:Saving checkpoints for 201 into trained_models/clust-model-02/model.ckpt.
INFO:tensorflow:global_step/sec: 63.1556
INFO:tensorflow:loss = 54775.6, step = 201 (1.583 sec)
INFO:tensorflow:Saving checkpoints for 301 into trained_models/clust-model-02/model.ckpt.
INFO:tensorflow:global_step/sec: 68.1381
INFO:tensorflow:loss = 54761.9, step = 301 (1.468 sec)
INFO:tensorflow:Saving checkpoints for 401 into trained_models/clust-model-02/model.ckpt.
INFO:tensorflow:global_step/sec: 61.6071
INFO:tensorflow:loss = 54757.0, step = 401 (1.623 sec)
INFO:tensorflow:Saving checkpoints for 501 into trained_models/clust-model-02/model.ckpt.
INFO:tensorflow:global_step/sec: 69.3657
INFO:tensorflow:loss = 54754.8, step = 501 (1.441 sec)
INFO:tensorflow:Saving checkpoints for 601 into trained_models/clust-model-02/model.ckpt.
INFO:tensorflow:global_step/sec: 63.9969
INFO:tensorflow:loss = 54753.5, step = 601 (1.563 sec)
INFO:tensorflow:Saving checkpoints for 701 into trained_models/clust-model-02/model.ckpt.
INFO:tensorflow:global_step/sec: 64.3657
INFO:tensorflow:loss = 54752.8, step = 701 (1.554 sec)
INFO:tensorflow:Saving checkpoints for 801 into trained_models/clust-model-02/model.ckpt.
INFO:tensorflow:global_step/sec: 66.5736
INFO:tensorflow:loss = 54752.3, step = 801 (1.502 sec)
INFO:tensorflow:Saving checkpoints for 901 into trained_models/clust-model-02/model.ckpt.
INFO:tensorflow:global_step/sec: 66.6998
INFO:tensorflow:loss = 54752.0, step = 901 (1.499 sec)
INFO:tensorflow:Saving checkpoints for 1000 into trained_models/clust-model-02/model.ckpt.
INFO:tensorflow:Loss for final step: 54751.7.
.......................................
Experiment finished at 18:48:57

Experiment elapsed time: 17.625221 seconds

5. Perform Predictions (Assign Instance to Clusters)


In [12]:
### evaluate function is broken! as well as score function
#estimator.evaluate(input_fn=test_input_fn)
#estimator.score(input_fn=test_input_fn)

train_input_fn = lambda: csv_input_fn(
            train_data_files,
            num_epochs=1,
            batch_size=1500
        )

test_input_fn = lambda: csv_input_fn(
            test_data_files,
            mode=tf.estimator.ModeKeys.EVAL,
            num_epochs=1,
            batch_size=500
        )

estimator =  create_estimator(run_config, hparams)

train_assignments = list(estimator.predict_cluster_idx(input_fn=train_input_fn))
test_assignments = list(estimator.predict_cluster_idx(input_fn=test_input_fn))


INFO:tensorflow:Using config: {'_task_type': None, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x120f46978>, '_master': '', '_num_ps_replicas': 0, '_num_worker_replicas': 0, '_environment': 'local', '_is_chief': True, '_evaluation_master': '', '_tf_config': gpu_options {
  per_process_gpu_memory_fraction: 1
}
, '_tf_random_seed': 19850610, '_save_summary_steps': 100, '_save_checkpoints_secs': None, '_log_step_count_steps': 100, '_session_config': None, '_save_checkpoints_steps': 100, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': 'trained_models/clust-model-02'}

Estimator Type: <class 'tensorflow.contrib.learn.python.learn.estimators.kmeans.KMeansClustering'>


* data input_fn:
================
Input file(s): ['data/train-data.csv']
Batch size: 1500
Epoch Count: 1
Mode: train
Shuffle: False
================

INFO:tensorflow:Restoring parameters from trained_models/clust-model-02/model.ckpt-1000

* data input_fn:
================
Input file(s): ['data/test-data.csv']
Batch size: 500
Epoch Count: 1
Mode: eval
Shuffle: False
================

INFO:tensorflow:Restoring parameters from trained_models/clust-model-02/model.ckpt-1000

In [13]:
import pandas as pd
import numpy as np

train_df = pd.read_csv(train_data_files[0], header=None, index_col=0)
test_df = pd.read_csv(test_data_files[0], header=None, index_col=0)


fig = plt.figure(figsize=(20,10))
ax = fig.add_subplot(121, projection='3d')

ax.scatter(train_df.iloc[:,0], train_df.iloc[:,1], train_df.iloc[:,2], c=train_assignments, marker='o')

ax = fig.add_subplot(122, projection='3d')

ax.scatter(test_df.iloc[:,0], test_df.iloc[:,1], test_df.iloc[:,2], c=test_assignments, marker='o')
plt.show()



In [14]:
clusters = estimator.clusters()
print("Cluster Centriods:")
print("==================")
print(clusters)


Cluster Centriods:
==================
[[ 7.68787766  2.15341544 -1.32305241]
 [ 8.83051014 -0.61770749  8.45560646]
 [-2.39564013 -4.19334888 -6.33953762]]

Serving via the Saved Model


In [15]:
export_dir = model_dir + "/export"

estimator.export_savedmodel(
    export_dir_base = export_dir,
    serving_input_fn = csv_serving_input_fn,
    as_text=False
)


INFO:tensorflow:Restoring parameters from trained_models/clust-model-02/model.ckpt-1000
INFO:tensorflow:Assets added to graph.
INFO:tensorflow:No assets to write.
INFO:tensorflow:SavedModel written to: b"trained_models/clust-model-02/export/temp-b'1510080539'/saved_model.pb"
Out[15]:
b'trained_models/clust-model-02/export/1510080539'

In [16]:
%%bash

MODEL_NAME='clust-model-02'
LAST=$(ls trained_models/${MODEL_NAME}/export | tail -1)
SAVE_MODEL_DIR=trained_models/$MODEL_NAME/export/$LAST
ls $SAVE_MODEL_DIR

gcloud ml-engine local predict --model-dir=$SAVE_MODEL_DIR --text-instances='data/new-data.csv'


saved_model.pb
variables
ERROR: (gcloud.ml-engine.local.predict) RuntimeError: Bad magic number in .pyc file


In [ ]: